#!/bin/bash

set -e

export https_proxy=http://10.10.15.2:3128

. util.sh

# local aliases are not enabled default on remote machine
shopt -s expand_aliases
# for debug and testing
alias sed_n="sed -n"
alias sed_i="sed -i"

ip=`ip a | grep 'inet ' | grep bond | egrep -o "\b([0-9]{1,3}\.){3}[0-9]{1,3}\b" | sed -n '1p'`

if [ x$ip = "x" ]; then
    ip=`ip a | grep 'inet ' | grep eth0 | egrep -o "\b([0-9]{1,3}\.){3}[0-9]{1,3}\b" | sed -n '1p'`
fi

cluster_name=""
seeds=""
dc=""
rack=""

user_name=""
keyspace_name=""
dc_replicas=""

hikv_admin_pw=""
user_pw=""

# default date, used in log command
cur_date=`date -d "0 day ago"`
date_marker=${cur_date:4:6}
log_suffix=.log
log_path="/tmp/hikv.log"

verbose=false

# for test on machines that don't have SSD or bond0
ignore_bond0=false
ignore_ssd=false
need_verify=true

# used in perf
sleep_num=10

# used in removenode
host_id=""

examples() {
    echo -e "\nExamples:\n"
    echo -e "change hikv settings such as cluster name, listen address:"
    echo -e "\t$0 setconfig --cluster-name test4 --dc dc4 --rack rack4 --seeds 2.2.2.5"
    echo -e "\t$0 setconfig --cluster-name test4 --dc dc4 --rack rack4 --seeds \"2.2.2.5,1.1.1.1\""
    echo -e "init hikv, including creat hikv_admin, create initial keyspace"
    echo -e "\t$0 account --keyspace ks1 --user user1 --dc-replicas "'bj':1, 'sh':2" -d"
    echo "-e "perform profiling for 10 seconds"
    echo "-e "\t$0 perf --sleep 10"
}
usage() {
    #echo -e "`basename $0` <subcommand> <options>"
    echo -e "  subcommand:"
    echo -e "\tstart                 start the current node"
    echo -e "\tstop                  stop the current node"
    echo -e "\trestart               restart the current node"
    echo -e "\tupgrade               stop the current node"
    echo -e "\texpand                add mores node"
    echo -e "\tshrink                remove existing nodes"
    echo -e "\tclear/clr             remove all data of one keyspace"
    echo -e "\tsetconfig/scfg        set some configurations in yaml files"
    echo -e "\taccount               create hikv_admin/passwd, create initial keyspace and so on"
    echo
    echo -e "\tinstall               install packages of hikv, node_exporter"
    echo -e "\tinstall-debuginfo     install hikv for debugging"
    echo -e "\tinstall-fg            install FlameGraph tools"
    echo -e "\tps                    check pid of hikv, node_exporter"
    echo -e "\tversion/ver           show hikv version"
    echo -e "\tgetconfig/gcfg        inspect some important configurations"
    echo -e "\tsanity/san            check whether setup requirements are met"
    echo -e "\terror/err             check log to find potential exceptions or errors in /var/log/messages"
    echo -e "\tlog                   behave like 'tail -f /var/log/messages'"
    echo -e "\ttail                  behave like 'tail -f /var/log/messages'"
    echo -e "\tperf                  use perf for profiling, such as generate FlameGraph"
    echo -e "\tbenchmark/bm          perform a micro-benchmark on test nodes"
    echo
    echo -e "\tstatus/st             same as 'nodetool status'"
    echo -e "\tcleanup               same as 'nodetool cleanup'"
    echo -e "\trepair/st             same as 'nodetool repair'"
    echo -e "\tremovenode/st         same as 'nodetool removenode'"
    echo
    echo -e "  options(Only for some commands):"
    echo -e "\t--cluster-name [name]                Cluster Name in scylla.yaml"
    echo -e "\t--dc [dc]                            dc property in cassandra-rack.yaml, used in setconfig command"
    echo -e "\t--rack [rack]                        rack property in cassandra-rack.ymal, used in setconfig command"
    echo -e "\t--seeds [seed_list]                  seeds property in cassandra-rack.yaml, separated by comma, muste be quoted"
    echo -e "\t--keyspace [keyspace_name]           Keyspace you want to operate"
    echo -e "\t--user [user_name]                   user name, used in account command"
    echo -e "\t--dc-replicas [replica_distribution] set which dc has how many replicas, such as \"'bj': 3, 'sh':3\", as keyspaces settings"

    echo -e "\t--date [days]                        how many days ago you want to inspect log for"
    echo -e "\t--sleep [num]                        sleep num seconds, such as performing perf"
    echo -e "\t--ignore-bond0                       DONOT exit even though network is not configured with bonding"
    echo -e "\t--ignore-ssd                         DONOT exit even though disk is not SSD type"
    echo -e "\t--host-id [host_id]                  remove host_id from cluster"
    echo -e "\t-v                                   verbose mode"
}

if [ $# = 0 ] || [ "$1" = "-h" ]; then
    usage
    exit 0
fi

subcmd=$1
shift
command_opt="$@"

# scylla_setup probably has done this
write_into_fstab() {
    lsblk -fl | grep md0 > /dev/null || exit_with_message "No md0"
    cat /etc/fstab | egrep 'scylla.*xfs'
    if [ $? != 0 ]; then
        uuid=`lsblk -fl | grep md0 | uniq | awk -F ' ' '{ print $3 }'`
        fstab_entry="UUID=$uuid /var/lib/scylla xfs noatime 0 0"
        echo "fstab_entry: $fstab_entry"
        echo $fstab_entry >> /etc/fstab
    fi
}

# It's not easy to find ununsed disks and do we need to double check ?
get_unused_disks() {
    lshw -C disk -short | grep '/dev/' | awk -F ' ' '{print $2}' | grep -v sda | tr '\n' ','
}

sanity_check_before_setup() {
    # lsblk, lshw
    yum -y install util-linux lshw openssl wget > /dev/null 2>&1

    lshw -C disk -short | egrep -i 'Intel SSD' || {
        if [ $ignore_ssd = "false" ]; then
            # After doing this, there'll be 'Intel SSD' in Description field
            echo "converting into Intel SSD ..."
            for i in {1..7};do
                /opt/MegaRAID/MegaCli/MegaCli64 -CfgLdDel -L${i} -a0;
            done && /opt/MegaRAID/MegaCli/MegaCli64 -AdpSetProp -EnableJBOD -1 -a0
        fi
    }
    # just check xfs
    #lsblk -f -l | egrep -i 'md0.*xfs.*scylla' ||
    #    if [ $ignore_ssd = "false" ]; then
    #        exit_with_message "No raid, or not xfs, or not mount into /var/lib/scylla";
    #    fi
    lsblk -f -l | grep -v sda | grep sd || exit_with_message "No other disks"
    ip a | grep bond0 || if [ $ignore_bond0 = "false" ];then exit_with_message "NO bond0"; fi

    # root partition should be mounted at /dev/sda
    mount | grep 'on / ' | grep sda || exit_with_message "/ is not on /dev/sda"
}

integrity_check_after_setup() {
    echo "integrity check after setup ..."
    which openssl > /dev/null 2>&1 || exit_with_message "openssl not installed"
    which scylla > /dev/null 2>&1 || exit_with_message "hikv not installed"
    which nodetool > /dev/null 2>&1 || exit_with_message "node_tool not installed"
    which cqlsh > /dev/null 2>&1 || exit_with_message "cqlsh not installed"
    which node_exporter > /dev/null 2>&1 || exit_with_message "node_exporter not installed"

    [ -e /etc/scylla/scylla.yaml ] || exit_with_message "scylla.yml missing"
    [ -e /etc/scylla/cassandra-rackdc.properties ] || exit_with_message "cassandra-rackdc.properties missing"
}

# inspect some important configurations
get_config() {
    set -x
    cat /etc/scylla/scylla.yaml |
        egrep 'cluster_name|seeds:|listen_address:|rpc_address:|endpoint_snitch:|datalog_total_space_in_mb:|experimental:|authenticator:|authorizer:' 
    cat /etc/scylla/cassandra-rackdc.properties | grep -E 'dc=|rack=|local'
    cat /etc/sysconfig/scylla-server | grep -E 'SCYLLA_ARGS'
    cat /usr/lib/systemd/system/scylla-server.service |
        grep -E 'ExecStart|StandardOutput|StandardError|Syslog'
    cat /etc/scylla.d/cpuset.conf | grep CPUSET
    cat /etc/scylla.d/io.conf
    cat /etc/systemd/coredump.conf
}

whether_change_settings() {
    flag=$1
    scylla_config_file=/etc/scylla/scylla.yaml
    dc_rack_file=/etc/scylla/cassandra-rackdc.properties

    # Used to see the changes
    output=/dev/stdout

    if [ $flag = false ]; then
        # Just preview the changes
        [ x$cluster_name != x ] &&
            sed_n "s;\(cluster_name:\).*;\1 \"$cluster_name\";w $output" $scylla_config_file
        [ x$seeds != x ] &&
            sed_n "s;\(seeds:\).*;\1 \"$seeds\";w $output" $scylla_config_file

        [ x$dc != x ] &&
            sed_n "s;.*\(dc=\).*;\1$dc;w $output" $dc_rack_file
        [ x$rack != x ] &&
            sed_n "s;.*\(rack=\).*;\1$rack;w $output" $dc_rack_file

        sed_n "s;\(endpoint_snitch:\).*;\1 GossipingPropertyFileSnitch;w $output" $scylla_config_file
        sed_n "s;.*[^_]\(authenticator:\).*;\1 PasswordAuthenticator;w $output" $scylla_config_file
        sed_n "s;.*\(authorizer:\).*;\1 CassandraAuthorizer;w $output" $scylla_config_file

        sed_n "s;\(listen_address:\).*;\1 $ip;w $output" $scylla_config_file
        sed_n "s;\(rpc_address:\).*;\1 $ip;w $output" $scylla_config_file
        sed_n "s;\(api_address:\).*;\1 $ip;w $output" $scylla_config_file
    elif [ $flag = true ]; then
        # Commit the changes
        [ x$cluster_name != x ] &&
            sed_i "s;\(cluster_name:\).*;\1 \"$cluster_name\";" $scylla_config_file
        [ x$seeds != x ] &&
            sed_i "s;\(seeds:\).*;\1 \"$seeds\";" $scylla_config_file

        [ x$dc != x ] &&
            sed_i "s;.*\(dc=\).*;\1$dc;" $dc_rack_file
        [ x$rack != x ] &&
            sed_i "s;.*\(rack=\).*;\1$rack;" $dc_rack_file

        sed_i "s;\(endpoint_snitch:\).*;\1 GossipingPropertyFileSnitch;" $scylla_config_file
        sed_i "s;.*[^_]\(authenticator:\).*;\1 PasswordAuthenticator;" $scylla_config_file
        sed_i "s;.*\(authorizer:\).*;\1 CassandraAuthorizer;" $scylla_config_file

        sed_i "s;\(listen_address:\).*;\1 $ip;" $scylla_config_file
        sed_i "s;\(rpc_address:\).*;\1 $ip;" $scylla_config_file
        sed_i "s;\(api_address:\).*;\1 $ip;" $scylla_config_file

    fi
}

set_config() {
    # perform preconditions check before setting configurations
    integrity_check_after_setup

    whether_change_settings false
    echo -n "Are these what you want ? [N]/y : "
    read verify
    if [ x$verify = x ] || [ $verify = 'N' ] || [ $verify = 'n' ]; then
        exit_with_message "There may be something wrong with setting-changing rule,check it"
    fi
    echo_with_color "changing hikv configurations ..."
    whether_change_settings true
}

# All Operations need to be executed once across the cluster !
init_account_and_keyspace() {
    integrity_check_after_setup
    pidof scylla || exit_with_message "scylla not started"
    cqlsh -u cassandra -p cassandra $ip 9042 -e 'describe keyspaces'
    if [ $? = 0 ];then
        [ x$user_name = x -o x$keyspace_name = x -o "x$dc_replicas" = x ] && exit_with_message "some parameters empty"
        printf "parameters summary:\n\tkeyspace=%s\n\tuser=%s\n\tdc=%s\n" $keyspace_name $user_name $dc_replicas

        hikv_admin_pw=`openssl rand -base64 12`
        hikv_user_pw=`openssl rand -base64 12`
        alter_ks_system_auth_stmt="alter keyspace system_auth with replication ={ 'class' : 'EverywhereStrategy' };"
        create_admin_stmt="create user hikv_admin with password '$hikv_admin_pw' superuser;"
        create_user_stmt="create user $user_name with password '$hikv_user_pw';"
        create_ks_stmt="create keyspace $keyspace_name with replication = { 'class': 'NetworkTopologyStrategy', $dc_replicas };"
        grant_select_stmt="grant select on keyspace $keyspace_name to $user_name;"
        grant_modify_stmt="grant modify on keyspace $keyspace_name to $user_name;"

        echo_verbosely "`printf "%s\n%s\n%s\n%s\n%s" "$alter_ks_system_auth_stmt"\
            "$create_admin_stmt" "$create_user_stmt" "$create_ks_stmt" "$grant_perm_stmt"`"

        set -x
        # expression after -e must be quoted
        cqlsh -u cassandra -p cassandra $ip 9042 -e "$alter_ks_system_auth_stmt" &&
            cqlsh -u cassandra -p cassandra $ip 9042 -e "$create_admin_stmt" &&
            cqlsh -u hikv_admin -p $hikv_admin_pw $ip 9042 -e "drop user cassandra;" &&
            cqlsh -u hikv_admin -p $hikv_admin_pw $ip 9042 -e "$create_user_stmt" &&
            cqlsh -u hikv_admin -p $hikv_admin_pw $ip 9042 -e "$create_ks_stmt" &&
            cqlsh -u hikv_admin -p $hikv_admin_pw $ip 9042 -e "$grant_select_stmt" &&
            cqlsh -u hikv_admin -p $hikv_admin_pw $ip 9042 -e "$grant_modify_stmt"

        if [ $? = 0 ]; then
            echo_with_color "[Password info] hikv_admin password: $hikv_admin_pw, $user_name password: $hikv_user_pw"
        else
            echo_with_color "[Account init failed] hikv_admin password: $hikv_admin_pw, $user_name password: $hikv_user_pw"
        fi
    fi
}

run_scylla_setup_scripts() {
    scylla_setup --disks $(./datadisks) --nic bond0 --setup-nic --no-selinux-setup --no-bootparam-setup \
        --no-ntp-setup --no-coredump-setup --no-version-check --no-node-exporter --no-cpuscaling-setup --no-fstrim-setup
}

install_node_exporter() {
    [ -e /usr/local/bin/node_exporter ] || wget -q 'http://10.15.191.115/public.php?service=files&t=21ab91e45a0f7df8ca2d4cce073242b7&download' -O /usr/local/bin/node_exporter
    chmod a+x /usr/local/bin/node_exporter

    yum -y install supervisor
    cat > /etc/supervisord.d/node_exporter.ini <<EOF
[program:node_exporter]
command=/usr/local/bin/node_exporter
autorestart=true
EOF
    systemctl enable supervisord.service
    chmod 0644 /etc/supervisord.d/node_exporter.ini
    systemctl restart supervisord.service
}

install_hikv() {
    yum -y remove abrt qiyi-cdn
    yum -y install qiyi-cloud-service
    yum -y install cs-db-repo
    yum -y install qiyi-epel

    yum -y install hikv

    systemctl stop fstrim.timer
    systemctl disable fstrim.timer

    install_node_exporter

    cat > /etc/systemd/coredump.conf <<EOF
[Coredump]
Storage=none
Compress=no
ProcessSizeMax=1G
ExternalSizeMax=1G
EOF
    if ! lsblk -f -l | egrep -i 'md0.*xfs.*scylla'; then
        unused_disks=`get_unused_disks`
        scylla_setup --disks ${unused_disks%,} --nic bond0 --setup-nic --no-selinux-setup --no-bootparam-setup \
            --no-ntp-setup --no-coredump-setup --no-version-check --no-node-exporter --no-cpuscaling-setup --no-fstrim-setup
    else
        echo "/var/lib/scylla already mounted on md0, performing io setup ..."
        scylla_io_setup
    fi
    echo "hikv installed and scylla_setup configured. Next you may configure the seed,cluster name, listen_address and so on."
}

install_hikv_debuginfo() {
    echo "$0"
}


start_hikv() {
    if ! pidof node_exporter; then
        node_exporter > /dev/null 2>&1 &
    fi

    if ! pidof scylla; then
        ls -sh /usr/bin/scylla
        systemctl start scylla-server
        # Must stop here until server is completely started !
        while true; do
            if systemctl status scylla-server.service  | grep 'listening for CQL'; then
               break
           else
               tail /var/log/messages | grep 'scylla:'
               sleep 1
            fi
        done
    fi
}

check_backtrace() {
    echo_with_color "Checking backtrace info ..."
    egrep -i 'backtrace:|scylla: 0x' $log_path

    # need to install scylla-debuginfo
    #egrep -i 'backtrace:|scylla: 0x' $log_path | grep '0x' | awk '{ print $6 }' | addr2line -Cfpi -e /usr/lib/debug/usr/bin/scylla.debug

}
check_anomaly() {
    egrep "${date_marker}.*scylla" /var/log/messages > $log_path
    check_backtrace
}

clear_dir() {
    if [ x$keyspace_name = x ]; then
        rm -rf /var/lib/scylla/commitlog/*
        rm -rf /var/lib/scylla/data/*
    else
        rm -rf /var/lib/scylla/commitlog/$keyspace_name/*
        rm -rf /var/lib/scylla/data/$keyspace_name/*
    fi
}

update_hikv() {
   integrity_check_after_setup
   nodetool drain
   systemctl stop scylla-server
   yum update -y hikv
}

disable_auto_bootstrap() {
    sed -n 's;SCYLLA_ARGS=.*\(--log-to-syslog\);SCYLLA_ARGS=\"--auto-bootstrap 0 \1;p' /etc/sysconfig/scylla-server
}
enable_auto_bootstrap() {
    sed -n 's;SCYLLA_ARGS=.*\(--log-to-syslog\);SCYLLA_ARGS=\"--auto-bootstrap 1 \1;p' /etc/sysconfig/scylla-server
}


# Nodetool command functions
repair() {
    nodetool repair --full --sequential
}
cleanup() {
    nodetool cleanup
}

benchmark() {
    echo
}

# enforce warning when important commands are issued !
warning() {
    if [ x$need_verify = x"false" ]; then return; fi
    cmd=$1
    name=`cat /etc/scylla/scylla.yaml | grep cluster_name | awk -F : '{ print $2 }'`
    # DONOT use read -p, since it cann't work via ssh
    echo -n "Do you really want to $cmd $ip in $name Cluster([N]/y) ? "
    read result
    if [ x$result = "x" ] || [ $result = "N" ] || [ $result = "n" ]; then
        exit
    fi
}

parse_opt() {
    while [ $# -gt 0 ]; do
        #if [[ $c != -* ]]; then continue; fi
        case $1 in
        -v)
            verbose="true"; shift
            ;;
        -y)
            need_verify="false"; shift
            ;;
        --keyspace)
            keyspace_name=$2; shift 2
            ;;
        --user)
            user_name=$2; shift 2
            ;;
        --dc-replicas)
            dc_replicas="$2"; shift 2
            ;;
        --cluster-name)
            cluster_name=$2; shift 2
            ;;
        --dc)
            dc=$2; shift 2
            ;;
        --seeds)
            seeds=$2; shift 2
            ;;
        --rack)
            rack=$2; shift 2
            ;;
        --sleep)
            sleep_num=$2; shift 2
            ;;
        --date)
            date_num=$2
            if [ x$date_num = x ]; then date_num=0; fi
            cur_date=`date -d "$date_num day ago"`
            # used to search the specified date on scylla log
            date_marker=${cur_date:4:6}
            log_suffix=`date -d "$date_num day ago" +%m-%d`.log
            log_path="/tmp/hikv-$log_suffix"
            shift 2
            ;;
        --ignore-bond0)
            ignore_bond0=true; shift
            ;;
        --ignore-ssd)
            ignore_ssd=true; shift
            ;;
        --host-id)
            host_id=$2; shift 2
            ;;
        *)
            exit_with_message "invalid parameter:$1"
            ;;
        esac
    done
    echo_verbosely "keyspace_name=$keyspace_name, user_name=$user_name, cluster_name=$cluster_name, dc=$dc, rack=$rack, seeds=$seeds"
}

main() {
    cmd=$1
    shift
    #echo "opt:$@"
    case $cmd in
    sanity|san)
        echo_with_color "sanity checking ..."
        sanity_check_before_setup
        ;;
    setconfig|scfg)
        echo_with_color "changing essential parameters for hikv ..."
        set_config
        ;;
    account)
        echo_with_color "setting hikv account and create initial keyspace (This operation should be executed once across cluster) ..."
        init_account_and_keyspace
        ;;
    start)
        echo_with_color "starting hikv and node_exporter ..."
        warning start
        start_hikv
        ;;
    stop)
        warning stop
        echo_with_color "stopping hikv ..."
        systemctl stop scylla-server
        ;;
    restart)
        warning restart
        echo_with_color "restarting hikv ..."
        systemctl stop scylla-server
        start_hikv
        ;;
    expand)
        echo_with_color "adding $ip to cluster ... "
        echo "This process includes install packages, set configurations, and start, so you must give configurations parameters such as cluster_name,seeds"
        warning expand
        install_hikv
        set_config
        start_hikv
        ;;
    shrink)
        echo_with_color "removing $ip from cluster ..."
        nodetool decommission
        systemctl stop scylla-server
        ;;
    repair)
        echo_with_color "repairing sequentially ..."
        repair
        ;;
    cleanup)
        echo_with_color "cleanup ..."
        cleanup
        ;;
    removenode)
        [ x$host_id != x ] || exit_with_message "host_id not given"
        echo_with_color "remove host_id=$host_id from cluster on $ip ..."
        nodetool removenode $host_id
        ;;
    install|inst)
        echo_with_color "installing hikv and node_exporter..."
        install_hikv
        ;;
    install-debuginfo)
        echo_with_color "installing hikv debuginfo ..."
        install_hikv_debuginfo
        ;;
    install-fg)
        echo_with_color "installing FlameGraph tools ..."
        install_flamegraph
        ;;
    upgrade)
        echo_with_color "updating hikv version ..."
        update_hikv
        echo_with_color "Repair on every node is required ! After reparing, you may start"
        ;;
    clear|clr)
        echo_with_color "Dangerous ! clearing data directories ..."
        warning clear
        clear_dir
        ;;
    status|st)
        nodetool status
        systemctl status scylla-server
        ;;
    error|err)
        echo_with_color "checking potential error log on \"$cur_date\" on $ip ..."
        check_anomaly
        ;;
    tail)
        tail -f /var/log/messages | grep scylla
        ;;
    log)
        tail -n 100 /var/log/messages | grep scylla
        ;;
    perf)
        echo_with_color "profiling hikv for $sleep_num seconds ..."
        ./perf.sh "$@"
        echo "Now you may need fetch the svg manually"
        ;;
    ps)
        ps aux | egrep -v grep | egrep 'scylla|node_exporter'
        ;;
    version|ver)
        scylla --version
        ;;
    getconfig|gcfg)
        echo_with_color "inspecting some important configurations ..."
        get_config
        ;;
    benchmark|bm)
        echo_with_color "micro loadtesting $ip ..."
        benchmark
        ;;
    help|h)
        usage
        exit 0
        ;;
    *)
        exit_with_message "invalid command:$cmd"
    esac
}

parse_opt "$@"

for i in "$@"; do
    echo_verbosely paramter:$i
done

echo_with_color "running $subcmd on $ip ..."
main $subcmd $command_opt
